*****************************************************************
*This file cleans the response status of institutions from IPEDS*
*****************************************************************

global years1 1989 1990 1991 1993 1994 1995 
foreach i in $years1 {
cd "$raw_data_education/responseStatus"

* completions flags
do "$do_clean_education/IPEDS - do/do responseStatus/c`i'_ic.do"
keep unitid unitidx rstatus parchild
rename rstatus stat_c
rename unitidx idx_c
rename parchild prch_c
save "$clean_data_education/responseStatusComp_`i'.dta", replace
clear

* enrollment flags
do "$do_clean_education/IPEDS - do/do responseStatus/ef`i'_ic.do"
keep unitid rstatus unitidx parchild
rename rstatus stat_fa
rename unitidx idx_ef
rename parchild prch_ef
replace stat_fa = 5 if stat_fa == 3 | stat_fa == -1 // to make it homogeneous with the other years 
save "$clean_data_education/responseStatus_`i'.dta", replace
clear
}

global years2 2006 2007 2008 2013 2014 2015
foreach i in $years2 {
cd "$raw_data_education/responseStatus"

do "$do_clean_education/IPEDS - do/do responseStatus/flags`i'.do"
keep unitid stat_ic fte prch_ef idx_ef stat_c prch_c idx_c prch_c stat_ef pcc_f pcef_f
rename stat_ef stat_fa
save "$clean_data_education/responseStatus_`i'.dta", replace
clear
}

**** from the institution details origin 
cd "$raw_data_education/institutionDetails"

foreach i in 1989 1990 1991 {
do "$do_clean_education/IPEDS - do/do institutionDetails/ic`i'_hdr.do"
keep unitid respstat
rename respstat stat_ic 
merge 1:1 unitid using "$clean_data_education/responseStatus_`i'.dta", nogenerate // merging with the other response status
merge 1:1 unitid using "$clean_data_education/responseStatusComp_`i'.dta", nogenerate // merging with the other response status
gen year = `i'
save "$clean_data_education/responseStatus_`i'.dta", replace
rm "$clean_data_education/responseStatusComp_`i'.dta"
clear
}


foreach i in 1993 1994 1995 {
do "$do_clean_education/IPEDS - do/do institutionDetails/ic`i'_a.do"
keep unitid respstat parchild unitidx
rename (unitidx parchild respstat) (idx_ic prch_ic stat_ic)
merge 1:1 unitid using "$clean_data_education/responseStatus_`i'.dta", nogenerate // merging with the other response status
merge 1:1 unitid using "$clean_data_education/responseStatusComp_`i'.dta", nogenerate // merging with the other response status
gen year = `i'
save "$clean_data_education/responseStatus_`i'.dta", replace
rm "$clean_data_education/responseStatusComp_`i'.dta"
clear
}

foreach i in 1999 {
do "$do_clean_education/IPEDS - do/do institutionDetails/ic`i'hd.do"
keep unitid stat_ic prch_ic idx_ic cyactive act newid deathyr
gen year = `i'
save "$clean_data_education/responseStatus_`i'.dta", replace
clear
}

foreach i in 2000 2001 {
do "$do_clean_education/IPEDS - do/do institutionDetails/fa`i'hd.do"
keep unitid stat_ic act newid deathyr fte stat_fa pset4flg prch_ef idx_ef prch_c idx_c stat_c openpubl
gen year = `i'
save "$clean_data_education/responseStatus_`i'.dta", replace
clear
}

foreach i in 2006 2007 2008 2013 2014 2015 {
do "$do_clean_education/IPEDS - do/do institutionDetails/fa`i'hd.do"
keep unitid act newid deathyr pset4flg openpubl  
merge 1:1 unitid using "$clean_data_education/responseStatus_`i'.dta", nogenerate // merging with the other response status
gen year = `i'
save "$clean_data_education/responseStatus_`i'.dta", replace
clear
}

* append
use "$clean_data_education/responseStatus_1989.dta", clear
foreach i in 1990 1991 1993 1994 1995 1999 2000 2001 2006 2007 2008 2013 2014 2015 {
append using "$clean_data_education/responseStatus_`i'.dta"
rm "$clean_data_education/responseStatus_`i'.dta"
}
rm "$clean_data_education/responseStatus_1989.dta"

gen active = .
replace active = 1 if act == "A" | act == "A "
replace active = 2 if act == "C" | act == "C "
replace active = 3 if act == "D" | act == "D "
replace active = 4 if act == "G" | act == "G "
replace active = 5 if act == "I" | act == "I "
replace active = 6 if act == "M" | act == "M "
replace active = 7 if act == "N" | act == "N "
replace active = 7 if act == "N" | act == "N "
replace active = 8 if act == "O" | act == "O "
replace active = 9 if act == "P" | act == "P "
replace active = 10 if act == "Q" | act == "Q "
replace active = 11 if act == "R" | act == "R "
replace active = 12 if act == "S" | act == "S "
replace active = 13 if act == "U" | act == "U "
replace active = 14 if act == "W" | act == "W "
replace active = 15 if act == "X" | act == "X "
replace active = 16 if act == "Z" | act == "Z "

label define active 1 "active" 2"Combined, merged with another inst." ///
3"Delete - institution is out of business" 4"Perfect child campus - data reported with main campus, but is listed on college navigator" ///
5"Inactive" 6"death with data" 7"New, just added" 8"Out-of-scope - not within scope of univers" ///
9"Potential add" 10"Potential restore - might be restored" 11"restored" 12"Split into more then one inst." ///
13"Duplicate - UNITID previously assigned" 14"Wipe out - out-of-scope potential add" ///
15"Potential restore not within scope of IPEDS" 16" Universe III - out-of-scope"
label value active active 
label variable active "status of the institution"
drop act


replace deathyr = 1999 if deathyr == -2 & unitid == 127990 & year == 1999
replace deathyr = 1999 if deathyr == -2 & unitid == 215804 & year == 1999
replace deathyr = 1999 if deathyr == -2 & unitid == 369206 & year == 1999
replace deathyr = 1999 if deathyr == -2 & unitid == 373845 & year == 1999
replace deathyr = 1999 if deathyr == -2 & unitid == 376604 & year == 1999
replace deathyr = 1999 if deathyr == -2 & unitid == 406079 & year == 1999
replace deathyr = 2000 if deathyr == -2 & unitid == 432135 & year == 1999

* final cleanings
do "$do_clean_education/clean institutions - id.do"

* year group
gen yearGroup = 0
replace yearGroup = 1990 if year <= 1991 
replace yearGroup = 1994 if year >= 1993 & year <= 1995
replace yearGroup = 2000 if year >= 1999 & year <= 2001
replace yearGroup = 2007 if year >= 2006 & year <= 2008
replace yearGroup = 2014 if year >= 2013

save "$clean_data_education/responseStatus.dta", replace
clear


***** Years that we DO NOT use in the analysis
foreach i in 2002 2003 2004 2005 2009 2010 2011 2012 {
do "$do_clean_education/IPEDS - do/do institutionDetails/hd`i'.do"
keep unitid act newid deathyr
rename (act newid deathyr) (act_`i' newid_`i' deathyr_`i')
save "$clean_data_education/responseStatus_`i'.dta", replace
clear
}

* create a unique dataset for the years we DO NOT use
use "$clean_data_education/responseStatus_2002.dta", clear
foreach i in 2003 2004 2005 2009 2010 2011 2012 {
merge 1:1 unitid using "$clean_data_education/responseStatus_`i'.dta", nogenerate 
rm "$clean_data_education/responseStatus_`i'.dta"
}
save "$clean_data_education/responseStatusOtherYears.dta", replace
rm "$clean_data_education/responseStatus_2002.dta"

